package paper.pdf;

import java.io.IOException;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileInputStream;
import java.io.BufferedInputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.io.File;
import java.io.PrintWriter;


import java.util.regex.*;
import java.util.Iterator;
import java.util.Vector;

import org.pdfbox.exceptions.InvalidPasswordException;
import org.pdfbox.pdmodel.PDDocument;
import org.pdfbox.util.PDFText2HTML;

import paper.token.TT;
import paper.token.Token;
import paper.RegExpVoter;

import paper.parse.*;
import paper.db.BibTex;
import paper.db.Paper;
import paper.db.PDFFile;
import paper.MyPDFStripper;
import paper.parse.TokenStream;
import paper.parse.BibToken;
import paper.parse.AuthorParse;
import paper.pdf.Average;
import paper.batch.Batch;

import java.io.InputStream;
import java.lang.Process;
import java.lang.Runtime;

public class ExtractTitle implements Extractor {

    final static HMMatcher titleParser = new TitleParser();
    final static HMMatcher bibliographyParserWithKey = new BibliographyParser(true, true);
    final static HMMatcher bibliographyParserNoKey = new BibliographyParser(true,false);
    final static Pattern emptyPatt = Pattern.compile("^[\n\\.; ]*$");

    private Vector<Token> bibList;
    private Vector<Integer> bibLines;
    private Pattern bibStart;
    private TokenStream ts;

    public ExtractTitle() {
    }

    public PDFFile process(Vector<String> batchList, File f, PrintWriter txt) {
        PDDocument document = null;
        ts = new TokenStream(txt);
        try {
            BufferedInputStream iStream = new BufferedInputStream(new FileInputStream(f));
            document = PDDocument.load(iStream);
            iStream.close();
            MyPDFStripper stripper = new MyPDFStripper();
            stripper.setShouldSeparateByBeads(false);
            stripper.extractText(document, ts);
            ts.eof();
            System.out.println("Extract now");
            return extract(f, batchList);
        } catch (Exception e) {
            System.err.println("Ouch..." + e);
            StackTraceElement[] ste = e.getStackTrace();
            for (int i = 0; i < ste.length; i++) {
                System.err.println(ste[i].toString());
            }
        } finally {
            if (document != null) {
                try {
                    document.close();
                } catch (Exception e) {
                    System.err.println("More ouch..." + e);
                }
            }
        }
        return null;
    }

    public PDFFile extract(File source, Vector<String> batchList) {
//        System.out.println(ts.toString());
        Parse pt = titleParser.parse(ts);
        StringBuilder email = new StringBuilder() ;
        StringBuilder authors = new StringBuilder() ;
        StringBuilder title = new StringBuilder() ;
        StringBuilder how = new StringBuilder() ;
        pt.lift(authors, TT.AUTHOR, false);
        pt.lift(email, TT.EMAIL, false);
        pt.lift(title, TT.TITLE, false);
        pt.lift(how, TT.RUBBISH, false);
        String eClean = cleanUp(email);
        String aClean = cleanUp(authors);
        String tClean = cleanUp(title);

        boolean hasAuthor = !emptyPatt.matcher(aClean).matches();
        boolean hasTitle = !emptyPatt.matcher(tClean).matches();
        boolean hasHeader = hasAuthor || hasTitle;

        Paper be = BibTex.find(aClean,tClean);
        boolean newEntry = be == null;
        if (newEntry) {
            be = BibTex.newPaper(aClean, eClean, tClean, "", "");
        } else {
            be.deleteReferences();
            be.mergeInformation(aClean, eClean, tClean, "");
        }
        StringBuilder data = new StringBuilder(pt.toString().replaceAll("\n","<br />"));

        bibList = ts.bibList();
        findLines();                          // fill in bibLines, bibStart
        boolean hasCitations = parseReferences(be, data, batchList) > 0;  // get refs
        if (newEntry) {
            if (hasHeader || hasCitations) {
                be.commitEntry("Parsing " + source);
            } else {
                tClean = "";
                be = null; // fixme, remove from BibTex cache.
            }
        }
        if (be != null) {
            batchList.add(be.key);
        }

        return new PDFFile("", be, hasHeader || hasCitations, false, hasTitle,
                           data.toString(), aClean, tClean, eClean,
                           how.toString());
    }

    static class Citation {
        String author, title, details;
        double probability;
        Paper to;
        Citation(Paper to, double probability, String author,
                 String title, String details) {
            this.author = author;
            this.title = title;
            this.details = details;
            this.probability = probability;
            this.to = to;
        }
    }

    static class Line {
        double diff;
        int tokenIndex;
        int dir;

        Line(double diff, int index) {
            this.diff = diff;
            this.tokenIndex = index;
        }
    }

    static class HistogramEntry {
        double diff;
        int count;
        final double epsilon = 1;
        HistogramEntry next = null;

        HistogramEntry(double diff) {
            this.diff = diff;
            this.count = 0;
        }
        
        void addOne(double x) {
            if (x < diff+1 && x > diff-1) {
                count++;
            } else if (next != null) {
                next.addOne(x);
            } else {
                next = new HistogramEntry(x);
            }
        }
        
        double highest(double sofar, int count) {
            if (this.count > count) {
                if (next == null) {
                    return diff;
                }
                return next.highest(diff, this.count);
            } else {
                if (next == null) {
                    return sofar;
                }
                return next.highest(sofar, count);
            }
        }
    }

    private void findLines() {
        Vector<Line> lineDiffs = new Vector<Line>();
        bibLines = new Vector<Integer>();
        double currentIndent = 0;
        HistogramEntry hist = null;
        boolean first = true;
        int realStart = 0 ;

        int nl = 0;
        String likelyKey = "";
        for (int i = 0; i < bibList.size(); i++) {
            Token t = bibList.get(i);
            if (t.t == TT.EOL) {
                nl++;
            }
            if (nl == 2 &&
                t.t != TT.EOL &&
                t.t != TT.NOTLARGE &&
                t.t != TT.BOLD &&
                t.t != TT.NOTBOLD &&
                t.t != TT.ITALIC &&
                t.t != TT.NOTITALIC &&
                t.t != TT.FONTCHANGE) {
                likelyKey = ts.substring(t.from);
                while (likelyKey.charAt(0) == ' ' ||
                       likelyKey.charAt(0) == '\n' ) {
                    likelyKey = likelyKey.substring(1);
                }
                realStart = i;
                break;
            }
        }

        Pattern bibNumberPeriodStart = Pattern.compile("([0-9]+)\\.[ \n]*");
        Pattern bibSquareBracketStart = Pattern.compile("\\[[ \n]*");
        Pattern bibHashStart = Pattern.compile("#[0-9][0-9a-fA-F][ \n]*");
        Pattern bibEmptyStart = Pattern.compile("");

        if (bibNumberPeriodStart.matcher(likelyKey).lookingAt()) {
            bibStart = bibNumberPeriodStart;
            System.out.println("COUNTER.");
            int nref = 1;
            boolean startOfLine = false;
            for (int i = 0; i < bibList.size(); i++) {
                Token t = bibList.get(i);
                if (t.t == TT.EOL) {
                    startOfLine = true;
                }
//                System.out.println("" + t);
                if (startOfLine &&
                    t.t != TT.EOL &&
                    t.t != TT.NOTLARGE &&
                    t.t != TT.ITALIC &&
                    t.t != TT.NOTITALIC &&
                    t.t != TT.BOLD &&
                    t.t != TT.NOTBOLD &&
                    t.t != TT.FONTCHANGE) {
                    String r = t.getRawText();
                    Matcher m = bibStart.matcher(r);
                    if (m.matches()) {
                        int val = Integer.parseInt(m.group(1));
                        if (nref*0.9 - 2 < val && val < nref * 1.1 + 2) {
                            nref++;
                            bibLines.add(i);
                        }
//                    } else {
//                        System.out.println("-   " + t);
                    }
                    startOfLine = false;
                }
            }
            return;
        }
        if (bibSquareBracketStart.matcher(likelyKey).lookingAt()) {
            bibStart = Pattern.compile("\\[[^\\]]*\\]");
            for (int i = 0; i < bibList.size(); i++) {
                Token t = bibList.get(i);
                String r = t.getRawText();
                Matcher m = bibSquareBracketStart.matcher(r);
                if (m.matches()) {
                    bibLines.add(i);
                }
            }
            return;
        }            

        // At this stage we have not found a standard key. Probablye
        // something like Blah, F (1995), ...
        bibStart = null;

/*        
        if (!likelyKey.equals("")) {
            if (likelyKey.length() > 20) {
                likelyKey = likelyKey.substring(0,19);
            }
            System.out.println("REFERENCE " + likelyKey);
        }
*/

        double lastVSpace = 0;
        boolean atEOL = false;
        Average auth  = new Average(5,30) ;
        Average nonAuth  = new Average(5,30) ;
        for (int i = realStart; i < bibList.size(); i++) {
            Token t = bibList.get(i);
            if (t.t == TT.EOL) {
                if (atEOL) {
                    t.spacing += lastVSpace;
                }
                atEOL = true;
                lastVSpace = t.spacing;
            } else if (atEOL && t.t != TT.NOTLARGE &&
                       t.t != TT.ITALIC &&
                       t.t != TT.NOTITALIC &&
                       t.t != TT.BOLD &&
                       t.t != TT.NOTBOLD &&
                       t.t != TT.FONTCHANGE) {
                atEOL = false;
//                    System.out.println(" + " + lastVSpace + " " + t);
                if (t.t == TT.AUTHOR) {
                    auth.addIn(lastVSpace);
                } else {
                    nonAuth.addIn(lastVSpace);
                }
            }
        }
        double varAuth = auth.stddev(), varOther = nonAuth.stddev();
        double totalVariance = varAuth + varOther;
        double meanAuth = auth.mean(), meanOther = nonAuth.mean();
        double meanDiff = meanAuth - meanOther;
        System.out.println("Auth : " + auth.mean() + " " + auth.stddev());
        System.out.println("NONa : " + nonAuth.mean() + " " + nonAuth.stddev());
        if (meanDiff > totalVariance/2) {
            double split = ((meanAuth * varAuth + meanOther * varOther)
                            / totalVariance);
            bibLines.add(realStart);
            for (int i = realStart; i < bibList.size(); i++) {
                Token t = bibList.get(i);
                if (t.t == TT.EOL) {
                    if (t.spacing < 5 || t.spacing > 30) {
                        if (i != bibList.size()-1 && bibList.get(i+1).t == TT.AUTHOR) {
                            bibLines.add(i+1);
                        }
                    } else if (t.spacing > split) {
                        bibLines.add(i+1);
                    }
                }
            }
            return;
        }

        for (int i = 0; i < bibList.size(); i++) {
            Token t = bibList.get(i);
            if (t.t == TT.EOL) {
                double lineDiff = t.indent - currentIndent;
                if (first) {
                    first = false;
                } else {
                    if (hist == null) {
                        lineDiff = 0;
                        hist = new HistogramEntry(0);
                    } else {
                        if (lineDiff > 1 || lineDiff < -1) {
                            hist.addOne(Math.abs(lineDiff));
                        } else {
                            lineDiff = 0;
                        }
                    }
//                  System.out.println("" + t.indent + " " + currentIndent + " " +  lineDiff + " " + bibList.get(i-1) + " " + t.spacing);
                    lineDiffs.add(new Line(lineDiff, i));
                }
                currentIndent = t.indent;
            }
        }
        if (hist == null) {
            return;
        }
        Line badLine = null;
        double split = hist.highest(0,-1);
        double lastPositive = 0;
        for (int i = 0; i < lineDiffs.size(); i++) {
            Line line = lineDiffs.get(i);
            double absLineDiff = Math.abs(line.diff);
            if (absLineDiff < split +1 &&
                (line.diff > 0 ? line.diff > lastPositive-0.5 : true)) {
//                System.out.println("Split " + line.diff + " ok");
                if (line.diff > 1) {
                    lastPositive = line.diff;
                    line.dir = 1;
                } else if (line.diff < -1) {
                    line.dir = -1;
                } else {
                    line.dir = 0;
                }
                if (badLine != null) {
                    badLine.dir = 9999;
                    badLine = null;
                }
            } else {
//                System.out.println("----- " + line.diff + " badLine " + badLine);
                if (badLine != null) {
                    badLine.diff += line.diff;
                    lineDiffs.remove(i);
                    i -= 2;
                    badLine = null;
                } else {
                    badLine = line;
                }
            }
        }
        int pos = 0;
        Line lastBadLine = null;
        for (int i = 0; i < lineDiffs.size(); i++) {
            Line line = lineDiffs.get(i);
            if (line.dir == 9999) {
                lastBadLine = line;
            } else {
                pos += line.dir;
                if (pos < 0) {
                    if (lastBadLine != null) {
                        lastBadLine.dir = 1;
                        lastBadLine = null;
                    }
                    pos = 0;
                } else if (pos > 1) {
                    if (lastBadLine != null) {
                        lastBadLine.dir = -1;
                        lastBadLine = null;
                    }
                    pos = 1;
                }
            }
        }

        pos = 0;
        for (int i = 0; i < lineDiffs.size(); i++) {
            Line line = lineDiffs.get(i);
            pos += line.dir;
            if (pos < 0) {
                pos = 0;
            } else if (pos > 1) {
                pos = 1;
            }
            if (pos == 0) {
                if (line.tokenIndex != bibList.size() - 1) {
                    bibLines.add(line.tokenIndex+1);
                }
            }
        }
/*
        for (int i = 0; i < lineDiffs.size(); i++) {
            Line line = lineDiffs.get(i);
            System.out.println("" + line.dir + " " + bibList.get(line.tokenIndex+1));
        }
*/
    }
    
    public int parseReferences(Paper fromBe, StringBuilder data, Vector<String> batchList) {
        Vector<Citation> citations = new Vector<Citation>();
        int nCit = 0;
        int nDitched = 0;
        data.append("<table>");
        for (int i = 0; i < bibLines.size(); i++) {
            int lineStart = bibLines.get(i), lineEnd;
            if (i == bibLines.size() - 1) {
                lineEnd = bibList.size();
            } else {
                lineEnd = bibLines.get(i+1);
            }
            data.append("<tr valign='top'><td align='right'>" + (i+1) + ".</td>");
            if (lineEnd - lineStart > 200) {
                data.append("<td colspan='2'>Silly length... bye!</td></tr>");
                break;
            }
            Citation c = extractOneRef(lineStart, lineEnd, data);
            if (c != null) {
                citations.add(c);
                nDitched = 0;
                nCit++;
            } else {
                if (nDitched++ > 3) {
                    data.append("<td colspan='2'>Derailed...</td></tr>");
                    break;
                }
            }
            data.append("</tr>");
        }
        data.append("</table>");
        for (Iterator<Citation> i = citations.iterator();
             i.hasNext();) {
            Citation c = i.next();
            fromBe.addCitation(c.to, c.probability, nCit, c.author, c.title, c.details);
            batchList.add(c.to.key);
        }
        return nCit;
    }
    

    public Citation extractOneRef(int firstToken, int lastToken, StringBuilder data) {
        BibToken bt = new BibToken(bibList, firstToken, lastToken, bibStart, ts);
        Parse pt = (bibStart == null?
                    bibliographyParserNoKey.parse(bt):
                    bibliographyParserWithKey.parse(bt));

        if (pt.prob < 1e-20) {
            data.append("<td>Improbable</td><td>" + pt + "</td>");
            return null;
        }

        StringBuilder a = new StringBuilder();
        StringBuilder t = new StringBuilder();
        StringBuilder h = new StringBuilder();
        pt.lift(a, TT.AUTHOR, false);
        pt.lift(t, TT.TITLE, false);
        pt.lift(h, TT.RUBBISH, false);
        String aClean = cleanUp(a);
        String tClean = cleanUp(t);
        String yClean = "";
        if (tClean.length() < 5 || aClean.length() < 2) {
            data.append("<td>Too short</td><td>" + pt + "</td>");
            return null;
        }
        Pattern year = Pattern.compile("\\b(19|20)[0-9][0-9]\\b");
        String entry = ts.substring(bibList.get(firstToken).from, bibList.get(lastToken-1).from + bibList.get(lastToken-1).length);
        Matcher m = year.matcher(entry);
        if (m.find()) {
            yClean = entry.substring(m.start(), m.end());
        }
        Paper be = BibTex.find(aClean, tClean);
        if (be == null) {
            be = BibTex.newPaper(aClean, "", tClean, yClean, "");
            be.commitEntry("Citation analysis");
        } else {
            be.mergeInformation(aClean, "", tClean, yClean);
        }
        data.append("<td>" + be.key + "</td>");
        data.append("<td>" + pt + "(" + pt.prob + ")" + "</td>");              // SHOUDL ALLOW TO GO UP TO NEXT KEY, NOT EOF!
        return new Citation(be, pt.prob, aClean, tClean, h.toString());
    }

    private String cleanUp(StringBuilder x) {
        int i;
        for (i = 0; i < x.length(); i++) {
            char c = x.charAt(i);
            if ("\n .:,()[]0123456789".indexOf(c) == -1) {
                break;
            }
        }
        if (i == x.length()) {
            return "";
        }
        int j;
        for (j = x.length()-1; j >= 0; j--) {
            char c = x.charAt(j);
            if ("\n .,:()[]0123456789".indexOf(c) == -1) {
                break;
            }
        }
        return x.substring(i, j+1);
    }

    
    public void sortAuthorsOut(String key) {
        AuthorParse.sortAuthors(key);
    }

}
